import pandas as pd
# import matplotlib.pyplot as plt
# from numpy import NaN,nan
import numpy as np
import datetime
from sklearn import ensemble
from sklearn.preprocessing import OneHotEncoder
import math
from utils import *
from xgboost import XGBRegressor, XGBRFRegressor

# def r2_score(y_pred, y_test):
#     y_pred = np.array(y_pred)
#     y_test = np.array(y_test)
#     y_test_mean = y_test.mean()
#     numerator = ((y_pred - y_test) * (y_pred - y_test)).sum()
#     denominator = ((y_test - y_test_mean) * (y_test - y_test_mean)).sum()
#     return 1- (numerator/denominator)




def date_to_nth_day(date, format='%Y%m%d'):
    dd = datetime.datetime.strptime(str(int(date)), format)
    dd_struct = dd.timetuple()
    return dd_struct.tm_year, dd_struct.tm_mon, dd_struct.tm_yday

def load_data(indir):
    df = pd.read_csv(indir + "/bd2019-weather-prediction-training-20190608.csv" )
    return df

def set_rain_value(rain):
    if rain >= 999990 or rain == 0:
        return 0
    v = math.log(rain) + 2.5
    if v < 1:
        return 0.33
    elif v < 3:
        return 0.5
    elif v < 6:
        return 0.8
    else:
        return 1

# def set_visibility(v):
#     if v >= 999990 :
#         return 0
#     elif v > 40000:
#         return 1
#     elif v > 20000:
#         return 0.8
#     elif v > 10000:
#         return 0.6
#     elif v > 5000:
#         return 0.4
#     elif v > 1000:
#         return 0.2
#     else:
#         return 0.1
def city_month(city, month):
    return city + str(month)

def load_avg_data(file_name):
    df = pd.read_csv(file_name)
    feature_engineering(df)
    #return df
    return df[(df['temperature'] < 999990) & (df['humidity'] < 999990)  ]

def city_month(city, month):
    return city + str(month)



def fill_date_info(df):
    series_dates = df['date'].apply(date_to_nth_day)

    ser_year = series_dates.apply(lambda x:x[0])
    ser_days = series_dates.apply(lambda x:x[2])
    ser_month = series_dates.apply(lambda x: x[1])
    df['year'], df['days'], df['month'], df['weeks'] = \
        ser_year, ser_days,ser_month, ser_days//7

    df['cm'] =  df.apply(lambda row: city_month(row['city'], row['month']), axis=1)
    df['cw'] =  df.apply(lambda row: city_month(row['city'], row['weeks']), axis=1)


def build_from_original(df, x_fields, y_field, conditions = ['df.date <= 20171201']):
    x_fields_condition = ["df['{}'].isnull() == False".format(x) for x in x_fields]
    df_train = df[(df[y_field] < 999990) &    # reject missing
                  (df[y_field].isnull().values == False) &  # reject NaN
                  eval('({})'.format(')&('.join(conditions))) &
                  eval('({})'.format(')&('.join(x_fields_condition) ))
                  ]
    x_part, y_part = df_train[x_fields], df_train[y_field]
    return x_part, y_part

def train_to_array(df_categories):
    ohe1 = OneHotEncoder(handle_unknown='ignore')
    ar = ohe1.fit_transform(df_categories).toarray()
    return ohe1, np.array(ar)


def test_to_array(df_categories, ohe):
    ar = ohe.transform(df_categories).toarray()
    return np.array(ar)

def merge_matrix(aa, bb):
    new_matrix = []
    assert len(aa) == len(bb)
    for i in range(len(aa)):
        new_matrix.append(np.hstack((aa[i], bb[i])))
    return np.array(new_matrix)

def run_train(x_train_df, y_train, category_fields, numeric_fields, model):
    df_category = x_train_df[category_fields]
    df_numeric = x_train_df[numeric_fields]
    onehot_encoder, x_ar = train_to_array(df_category)
    x_ar2 = np.array(df_numeric)
    print(x_ar.shape, x_ar2.shape , end =" > ")
    x_train = merge_matrix(x_ar, x_ar2)
    print(x_train.shape, y_train.shape)
    model.fit(x_train, y_train)
    return model, onehot_encoder


def run_test(x_test_df, y_test, category_fields, numeric_fields, model, onehot_encoder):
    df_category = x_test_df[category_fields]
    df_numeric = x_test_df[numeric_fields]
    x_ar = test_to_array(df_category, onehot_encoder)
    x_ar2 = np.array(df_numeric)
    print(x_ar.shape, x_ar2.shape, end=" > ")
    x_test = merge_matrix(x_ar, x_ar2)
    print(x_test.shape, y_test.shape)
    result = model.predict(x_test)
    score = model.score(x_test, y_test)
    x_test_df['predict'] = result
    x_test_df['ytest'] = y_test
    x_test_df[category_fields + ['ytest','predict']].to_csv("validation.csv", index=False)
    return result, score


def compare_models_fix_data(df, models, category_fields, numeric_fields, yfield):
    print("=========now :{}==========\n".format(yfield))
    x_train_df, y_train = build_from_original(df, category_fields+ numeric_fields, yfield, conditions = ['df.date <= 20171201'])
    print(len(x_train_df), len(y_train))
    x_test_df, y_test = build_from_original(df, category_fields+ numeric_fields, yfield, conditions = ['df.date > 20171201'])
    print(len(x_test_df), len(y_test))

    for model_select in models:
        model, encoder = run_train(x_train_df, y_train, category_fields, numeric_fields, model_select)
        result, score = run_test(x_test_df, y_test, category_fields, numeric_fields, model, encoder)
        print("{} \nscore:{}".format(repr(model_select), score))


# def print_rf_importance(forest, x_train):
#     importances = forest.feature_importances_
#     indices = np.argsort(importances)[::-1]
#     for f in range(x_train.shape[1]):
#         print("%2d) %-*s %f" % (f + 1, 30, feat_labels[indices[f]], importances[indices[f]]))

def compare_fields_fix_model(df, category_fields, numeric_fields, yfields, model_select):
    print("{}".format( repr(model_select)))
    for yfield in yfields:
        print("=========now :{}==========\n".format(yfield))
        nnfields = numeric_fields = list(set(numeric_fields) - set([yfield]))
        xfields = category_fields + nnfields
        x_train_df, y_train = build_from_original(df, xfields, yfield, conditions = ['df.days > 10'])
        print(len(x_train_df), len(y_train))
        x_test_df, y_test = build_from_original(df, xfields, yfield, conditions = ['df.days <= 10'])
        print(len(x_test_df), len(y_test))
        model, encoder = run_train(x_train_df, y_train, category_fields, nnfields, model_select)
        result, score = run_test(x_test_df, y_test, category_fields, nnfields, model, encoder)
        print("\tscore:{}".format( score))
        print()



if __name__ == "__main__":
    # model_decision_tree_regression = tree.DecisionTreeRegressor()
   # df = load_data("d:/sfxy191")
    df = load_avg_data('mod/train1.csv')
    from_days = FD
    window_size = WS
    span = SP

   # shift_fields = ['temperature', 'humidity', 'cloud']

    category_fields = ['year', 'month','city', 'station']
    numeric_fields = ['days','pressure', 'cloud', 'rain']


    yfields = ['temperature']

    model_random_forest_regressor = ensemble.RandomForestRegressor(n_estimators=20)
    model_adaboost_regressor = ensemble.AdaBoostRegressor(n_estimators=50)
    model_gradient_boosting_regressor = ensemble.GradientBoostingRegressor(n_estimators=100)
    model_bagging_regressor = ensemble.BaggingRegressor(n_estimators=20)
    model_xgboost1 = XGBRegressor()
    #model_xgboost2 = XGBRFRegressor()
    models = [model_random_forest_regressor, model_bagging_regressor]
   # compare_models_fix_data(df, models, category_fields, numeric_fields, yfields[0])
    #compare_models_fix_data(df, models, category_fields, numeric_fields, yfields[-1])
    compare_fields_fix_model(df, category_fields,numeric_fields, yfields, model_random_forest_regressor)